{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "data from https://www.kaggle.com/c/avazu-ctr-prediction/data\n",
    "\"\"\"\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "import tensorflow as tf\n",
    "\n",
    "path_data = \"./datasets/train.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(path_data, nrows=20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C1                int64\nbanner_pos        int64\nsite_category    object\napp_category     object\ndevice_id        object\nC16               int64\nC18               int64\nC19               int64\nC20               int64\ndtype: object\n"
     ]
    }
   ],
   "source": [
    "x = data[[\"C1\", \"banner_pos\", \"site_category\", \"app_category\", \"device_id\", \"C16\", \"C18\", \"C19\", \"C20\"]]\n",
    "item = data[\"id\"]\n",
    "y = data[\"click\"]\n",
    "print(x.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 2244)\n"
     ]
    }
   ],
   "source": [
    "les = [LabelEncoder() for _ in x.columns]\n",
    "[le.fit(x[col]) for le, col in zip(les, x.columns)]\n",
    "\n",
    "ohe = OneHotEncoder(sparse=False)\n",
    "num_x = np.concatenate([le.transform(x[col])[:, None] for le, col in zip(les, x.columns)], axis=1)\n",
    "ohe.fit(num_x)\n",
    "print(ohe.transform(num_x[:3]).shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "class MyOneHotEncoder(OneHotEncoder):\n",
    "    def __init__(self, data, sparse=True, n_values=\"auto\", categorical_features=\"all\",\n",
    "                 dtype=np.float64, handle_unknown='error'):\n",
    "        \n",
    "        self.les = [LabelEncoder() for _ in data.columns]\n",
    "        [le.fit(x[col]) for le, col in zip(self.les, data.columns)]\n",
    "        \n",
    "        super().__init__(n_values=n_values, categorical_features=categorical_features,\n",
    "                 dtype=dtype, sparse=sparse, handle_unknown=handle_unknown)\n",
    "        \n",
    "        self.fit(self.to_id(data))\n",
    "        self.n_col = self.n_values_.sum()\n",
    "    \n",
    "    def to_id(self, data):\n",
    "        id_data = np.concatenate([le.transform(data[col])[:, None] for le, col in zip(les, data.columns)], axis=1)\n",
    "        return id_data\n",
    "    \n",
    "    def sparse_transform(self, X):\n",
    "        self.sparse = True\n",
    "        return self.transform(self.to_id(X))\n",
    "    \n",
    "    def dense_transform(self, X):\n",
    "        self.sparse = False\n",
    "        return self.transform(self.to_id(X))\n",
    "    \n",
    "    @staticmethod\n",
    "    def to_tf_format(sparse_matrix):\n",
    "        sparse_m = ohe.sparse_transform(sparse_matrix).tocoo()\n",
    "        indices = np.mat([sparse_m.row, sparse_m.col]).transpose()\n",
    "        return tf.SparseTensorValue(indices, sparse_m.data, sparse_m.shape)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ohe = MyOneHotEncoder(x)\n",
    "# print(ohe.sparse_transform(x[:3]))\n",
    "# print(ohe.dense_transform(x[:3]).shape)\n",
    "# print(ohe.feature_indices_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  5.]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "x = tf.sparse_placeholder(tf.float32)\n",
    "y = tf.sparse_reduce_sum(x, axis=1)\n",
    "x_ = tf.sparse_tensor_to_dense(x)\n",
    "sess = tf.Session()\n",
    "indices = np.array([[0, 3], [1, 5]], dtype=np.int64)\n",
    "values = np.array([1.0, 5.0], dtype=np.float32)\n",
    "shape = np.array([2, 9], dtype=np.int64)\n",
    "print(sess.run(y, feed_dict={\n",
    "x: tf.SparseTensorValue(indices, values, shape)}))  # Will succeed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
